import asyncio
from playwright.async_api import async_playwright, BrowserType
import pymongo
import json
import gzip
import io
import time
import asyncio
from playwright.async_api import async_playwright, BrowserType
from asyncqt import QEventLoop
from PyQt5.QtWidgets import QApplication, QPushButton, QVBoxLayout, QWidget, QTextEdit
import sys
import time
import random
import pandas as pd
import tkinter as tk
from tkinter import scrolledtext

class BaseAsync:
    def __init__(self):
        self._playwright = None
        self._browser = None
        self._context = None
        self._pages = []  # 维护一个页面列表
        self.user_data_dir = 'C:/Users/Administrator/AppData/Local/Google/Chrome/User Data/Default'

        self.client = pymongo.MongoClient('localhost', 27017)
        self.db = self.client['pdd']
        self.collection_name=None
        self.mall_id = None

        self.seen_links = set()  # 初始化一个空集合，用于存储已经见过的链接


    async def setup_playwright(self):
        """Asynchronously setup Playwright instance."""
        self._playwright = await async_playwright().start()
        self._browser = await self._playwright.chromium.connect_over_cdp('http://localhost:9999')
        self._context = self._browser.contexts[0]
        # self._context = await self._browser.new_context()
        # 使用 launch_persistent_context 而不是 launch  user_data_dir=self.user_data_dir,
        # self._context = await self._playwright.chromium.launch_persistent_context(user_data_dir=self.user_data_dir,
        #                                                                           headless=False)

    async def wait_for_captcha_and_continue(self,page):
        """
        等待验证码出现，并在用户处理后继续执行。

        :param page: Playwright的Page对象
        """
        try:
            # 等待滑块验证码出现，设置超时时间，例如10秒
            await asyncio.wait_for(page.wait_for_selector('.slider-img'), timeout=2)
            await asyncio.sleep()
            input("验证码出现，请手动处理...")
            # 将浏览器窗口置顶


        except asyncio.TimeoutError:
            # 如果在超时时间内没有检测到验证码，正常继续流程
            pass


    async def new_open_close_page(self, url):
        page = await self._context.new_page()
        self._pages.append(page)  # 添加到页面列表
        await self.attach_network_listeners(page)  # 为新页面添加网络监听器
        try:
            await page.goto(url)
            await self.wait_for_captcha_and_continue(page)#等待验证码
            # 等待页面加载完成，可以使用 page.wait_for_load_state('networkidle') 或其他等待策略
            await page.evaluate(
                """
                () => {
                    return new Promise(resolve => {
                        if (document.readyState === 'complete') {
                            resolve(true);
                        } else {
                            const observer = new MutationObserver(() => {
                                if (document.readyState === 'complete') {
                                    resolve(true);
                                    observer.disconnect();
                                }
                            });
                            observer.observe(document, { childList: true, subtree: true });
                        }
                    });
                }
                """
            )
            await asyncio.sleep(random.randint(3, 5))  # 使用异步 sleep

        finally:
            # 关闭页面
            await page.close()
            if page in self._pages:
                self._pages.remove(page)
                # print("页面从列表中移除")


    async def modify_navigator_properties(self):
        """
        Modify properties on the navigator object to avoid detection.
        This script will run on every new page created in the context.
        """
        script = """
            Object.defineProperty(navigator, 'webdriver', {
            get: () => undefined
            });
            """
        await self._context.add_init_script(script)

    async def open_url(self, url):
        """Asynchronously open URL using Playwright and keep the browser open."""
        if not self._browser:
            await self.setup_playwright()
        if not self._context:
            self._context = await self._browser.new_context()
        page = await self._context.new_page()
        await page.goto(url)
        await self.wait_for_captcha_and_continue(page)  # 等待验证码
        await self.attach_network_listeners(page)  # 为新页面添加网络监听器
        # await self.modify_navigator_properties()
        self._pages.append(page)  # 添加到页面列表
        await self.attach_network_listeners(page)  # 为新页面添加网络监听器

    async def update_data_shop_good(self, json_data):
        """Get JSON data from data_src."""
        # if self.collection is None:
        #     print("请先选择一个店铺,刷新一下")
        #     return
        data_dict = json_data.get('result')
        # shop_good_lists = data_dict.get("goodsPropertyTexts")
        shop_good_lists = data_dict.get("goodsCardVOList")
        if shop_good_lists:
            for item in data_dict.get("goodsCardVOList", []):  # 遍历商店商品列表
                mall_encrypt_id = item.get("mallIdEncrypt")  # 获取当前商品的mallIdEncrypt
                self.mall_id = mall_encrypt_id
                # print("当前是商店页", self.mall_id)
                if self.collection.update_one(
                        {"mallIdEncrypt": mall_encrypt_id},
                        {"$addToSet": {"accumulatedGoodsIds": item}},  # 使用$addToSet来累积goodsId
                        # {"$addToSet": {goods_id: item}},  # 使用$addToSet来累积goodsId
                        upsert=True
                ):
                    print(f"商品列表：({item['goodsName']}) 数据更新成功")
                    # 这个是从json中提取的id拼接的，估计不安全
                    # good_url=f"https://pifa.pinduoduo.com/goods/detail/?gid={item['goodsId']}"
                    # print("商品详情页链接",good_url)
                    #
                    # await self.new_open_close_page(good_url)
        else:
            # property_mall_id = data_dict.get("mallCard", {}).get("mallId", None)
            # # print("当前是商品详情页mallid", property_mall_id)
            # if self.mall_id==property_mall_id:
            if self.collection.update_one(
                {"goodsId": data_dict["goodsId"]},  # 查询条件
                {"$set": data_dict},  # 要更新的字段及其值
                upsert=True):  # 如果没有匹配的文档，则插入新文档

                return True

    async def handle_page_before_unload(self, page):
        print("页面关闭111111111111111111111111111111111111")
        await asyncio.sleep(random.randint(1, 2))  # 使用异步 sleep
        try:
            # 检查页面类型
            url = await page.evaluate('() => window.location.href')
            print(f"当前页面类型: {url}")
            if url.startswith("https://pifa.pinduoduo.com/mall?mid="):
                print("当面关闭的是店铺页，请过会查看数据信息")
                # 异步数据库查询
                cursor = self.db[self.collection_name].find()
                documents = await cursor.to_list(length=None)
                # 将数据转换为DataFrame
                df = pd.DataFrame(documents)
                # 异步写入CSV文件
                async with open(f'{self.collection_name}.csv', mode='w') as f:
                    await f.write(df.to_csv(index=False))
        except Exception as e:
            print(f"页面已关闭，无法获取信息: {e}")


    async def export_all_collections_to_csv(self):
        # 获取数据库中所有集合的名称
        collection_names = self.db.list_collection_names()

        for collection_name in collection_names:
            # 从数据库中读取集合数据
            collection = self.db[collection_name]
            documents = list(collection.find())

            # 将数据转换为DataFrame
            df = pd.DataFrame(documents)

            # 将DataFrame写入CSV文件
            df.to_csv(f'{collection_name}.csv', index=False)

    async def attach_network_listeners(self, page):
        """
        Attach network request and response listeners to a given page.
        """
        def print_request(request):
            pass
            # print(f"请示页地址： {request}: {request.method} {request.url}")
            # print(f"Sending request on page {request}: {request.method} {request.url}")

        async def print_response(response):  #商品详情页
            # print(f"Received response on page {response}: {response.status} {response.url},{response.request.url}")
            shop_list = ["https://pifa.pinduoduo.com/pifa/supplier/queryMallInfo"]
            if response.url in shop_list:
                json_data = await response.json()

                mall_name = json_data['result']['mallName']
                print(mall_name)  # 输出: 左舒达益智专营店
                self.collection_name = mall_name
                self.collection = self.db[self.collection_name]
                self.is_collection = True
                # await asyncio.sleep(random.randint(1, 2))  # 使用异步 sleep

            good_list_shop = ["https://pifa.pinduoduo.com/pifa/supplier/queryGoodsList"]  # 商店商品列表

            if response.url in good_list_shop:
                # 假设响应内容为JSON格式，使用.json()方法提取
                json_data = await response.json()
                await self.update_data_shop_good(json_data)
                print("11商店商品列表更新")
                # await asyncio.sleep(random.randint(1, 2))  # 使用异步 sleep

                # 从这儿提取商品页面链接，应该更安全,每次只取新增的链接
                print(await page.evaluate('window.location.href'))
                await self.get_goods_links_and_open(page)
            # await asyncio.sleep(random.randint(1, 2))  # 使用异步 sleep
            good_list_infos = [
                "https://pifa.pinduoduo.com/pifa/goods/queryGoodsPropertyInfo",
                "https://pifa.pinduoduo.com/pifa/goods/queryGoodsDetail",
            ]  # 详情页
            if response.url in good_list_infos:
                # 假设响应内容为JSON格式，使用.json()方法提取
                json_data = await response.json()
                await self.update_data_shop_good(json_data)

        # await asyncio.sleep(random.randint(2, 3))  # 使用异步 sleep
        # page.on("request", print_request)   #暂停监视发送请求
        page.on("response", print_response)
        # 将此函数绑定到页面关闭事件
        # page.on("close", lambda p=page: asyncio.create_task(self.handle_page_close(p)))
        # page.on("unload", lambda p=page: asyncio.create_task(self.handle_page_before_unload(p)))

        # page.on("close", lambda: self._pages.remove(page) if page in self._pages else None)
        # page.on("response",process_mall_page)
        # page.on("popup", lambda popup_page: asyncio.create_task(self.attach_network_listeners(popup_page)))  # 监听新页面弹出
        # 监听新页面弹出，并添加到页面列表
        page.on("popup", lambda popup_page:asyncio.create_task(self.attach_network_listeners(popup_page)).add_done_callback(lambda fut:self._pages.append(
popup_page) if not fut.exception() else print("弹出窗口弹出失败")))
        # 监听页面内导航
        # page.on("framenavigated", lambda navigation: asyncio.create_task(on_frame_navigated(navigation)))




    async def get_goods_links_and_open(self, page):
        # 使用 page.evaluate() 执行 JavaScript 代码，获取所有 a.goods-link 元素的 href 属性
        links = await page.evaluate(
            """
            () => {
                let links = [];
                document.querySelectorAll('a.goods-link').forEach((element) => {
                    links.push(element.href);
                });
                return links;
            }
            """
        )

        # 创建一个空集合，用于存储新增的链接
        # new_links = set()

        for link in links:
            # 如果链接还没有被见过，就添加到 new_links 集合中
            if link not in self.seen_links:
                self.seen_links.add(link)  # 同时更新 seen_links 集合
                # 打开新页面
                await self.new_open_close_page(link)
                # await asyncio.sleep(random.randint(1, 2))  # 使用异步 sleep
                print(f"提取：{link}")
                # new_links.add(link)


        # return new_links

    async def teardown(self):
        """Asynchronously clean up resources manually when you're done."""
        while self._pages:  # 关闭所有页面
            page = self._pages.pop()
            await page.close()
        if self._context:
            await self._context.close()
        if self._browser:
            await self._browser.close()
        if self._playwright:
            await self._playwright.stop()

    async def monitor_network_traffic(self):
        """Asynchronously monitor network traffic across all managed pages."""
        pass  # 监控逻辑可以放在这里，但当前监听器已经足够处理跨页面的网络请求

    async def main(self):
        """Asynchronous entry point."""
        await self.open_url("https://pifa.pinduoduo.com/")


        print("Press Ctrl+C to stop monitoring and close the browser...")
        while True:  # 保持程序运行直到中断
            await asyncio.sleep(1)


async def run_example():
    your_instance = BaseAsync()
    try:
        await your_instance.main()
    finally:
        await your_instance.teardown()

asyncio.run(run_example())
